{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import confusion_matrix as cm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# base weight :mask_rcnn_chips_3_090417.h5\n",
    "test_1 = pd.read_csv('result/test_chips3_15.csv') # test 0917null.h5 on test\n",
    "test_2 = pd.read_csv('result/test_chips3_20.csv') # similar with above \n",
    "test_3 = pd.read_csv('result/test_chips3_25_reversed.csv')# test reversed.h5 on test \n",
    "test_4 = pd.read_csv('result/test_chips3_1009.csv')# test 1009.h5 (10 epoch training on data4) ( about 80% null empty) on test\n",
    "test_5 = pd.read_csv('result/test_chips3_data4_1012.csv')#test null train h5 on new generated data4 (precison 78%)\n",
    "test_6 = pd.read_csv('result/test_chips3_data4_1015.csv')#test 090417.h5 on data4 (precision 58%)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "def error_display(result,num):\n",
    "    y_true = result['gt_class']\n",
    "    y_pred = result['pre_class']\n",
    "    cmatrix = cm(y_true,y_pred)\n",
    "    \n",
    "    num_finechips = sum(cmatrix[0])\n",
    "    num_flawchips = len(result)-num_finechips\n",
    "    num_pre_finechips = sum(cmatrix[:,0])\n",
    "    num_pre_flawchips = len(result)-num_pre_finechips\n",
    "    \n",
    "    print('confusion matrix:\\n')\n",
    "    print(cmatrix)\n",
    "    \n",
    "    pres = []\n",
    "    recs = []\n",
    "    for i in range(4):\n",
    "        precison = cmatrix[i,i]/sum(cmatrix[i])*100\n",
    "        recall = cmatrix[i,i]/sum(cmatrix[:,i])*100\n",
    "        pres.append(precison)\n",
    "        recs.append(recall)\n",
    "        print('precision and recall on class%d ： %d%%   %d%% \\n'%(i,precison,recall))\n",
    "    \n",
    "    print('total validation samples num ：  %d'%len(result))\n",
    "    print('mean presicion ： %d%%'%(np.mean(pres)))\n",
    "    print('mean recall ： %d%%'%(np.mean(recs)))\n",
    "    print('mean ap ：  %d%% '% (np.mean(result['ap'])*100))\n",
    "    return "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "condusion matrix:\n",
      "\n",
      "[[1175   42   38    8]\n",
      " [  17 1355    6    4]\n",
      " [  34    4 1884    4]\n",
      " [  22   13   10 1323]]\n",
      "precision and recall on class0 ： 93%   94% \n",
      "\n",
      "precision and recall on class1 ： 98%   95% \n",
      "\n",
      "precision and recall on class2 ： 97%   97% \n",
      "\n",
      "precision and recall on class3 ： 96%   98% \n",
      "\n",
      "total validation samples num ：  5939\n",
      "mean presicion ： 96%\n",
      "mean recall ： 96%\n",
      "mean ap ：  76% \n"
     ]
    }
   ],
   "source": [
    "error_display(test_1,1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "condusion matrix:\n",
      "\n",
      "[[1197   31   30    5]\n",
      " [  36 1339    5    2]\n",
      " [  49    4 1870    3]\n",
      " [  26   14   12 1316]]\n",
      "precision and recall on class0 ： 94%   91% \n",
      "\n",
      "precision and recall on class1 ： 96%   96% \n",
      "\n",
      "precision and recall on class2 ： 97%   97% \n",
      "\n",
      "precision and recall on class3 ： 96%   99% \n",
      "\n",
      "total validation samples num ：  5939\n",
      "mean presicion ： 96%\n",
      "mean recall ： 96%\n",
      "mean ap ：  75% \n"
     ]
    }
   ],
   "source": [
    "error_display(test_2,2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "condusion matrix:\n",
      "\n",
      "[[1233   12   14    4]\n",
      " [  85 1293    4    0]\n",
      " [  86    8 1832    0]\n",
      " [  38   21   22 1287]]\n",
      "precision and recall on class0 ： 97%   85% \n",
      "\n",
      "precision and recall on class1 ： 93%   96% \n",
      "\n",
      "precision and recall on class2 ： 95%   97% \n",
      "\n",
      "precision and recall on class3 ： 94%   99% \n",
      "\n",
      "total validation samples num ：  5939\n",
      "mean presicion ： 95%\n",
      "mean recall ： 94%\n",
      "mean ap ：  73% \n"
     ]
    }
   ],
   "source": [
    "error_display(test_3,3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "condusion matrix:\n",
      "\n",
      "[[1060  119   43   41]\n",
      " [ 403  236  532  211]\n",
      " [ 156  198 1313  259]\n",
      " [ 159   34  497  678]]\n",
      "precision and recall on class0 ： 83%   59% \n",
      "\n",
      "precision and recall on class1 ： 17%   40% \n",
      "\n",
      "precision and recall on class2 ： 68%   55% \n",
      "\n",
      "precision and recall on class3 ： 49%   57% \n",
      "\n",
      "total validation samples num ：  5939\n",
      "mean presicion ： 54%\n",
      "mean recall ： 52%\n",
      "mean ap ：  35% \n"
     ]
    }
   ],
   "source": [
    "error_display(test_4,4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "condusion matrix:\n",
      "\n",
      "[[3613  237   79   28]\n",
      " [ 138   17  128    0]\n",
      " [  63   77  214   67]\n",
      " [  17   43  148   75]]\n",
      "precision and recall on class0 ： 91%   94% \n",
      "\n",
      "precision and recall on class1 ： 6%   4% \n",
      "\n",
      "precision and recall on class2 ： 50%   37% \n",
      "\n",
      "precision and recall on class3 ： 26%   44% \n",
      "\n",
      "total validation samples num ：  4944\n",
      "mean presicion ： 43%\n",
      "mean recall ： 45%\n",
      "mean ap ：  5% \n"
     ]
    }
   ],
   "source": [
    "error_display(test_5,5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "condusion matrix:\n",
      "\n",
      "[[2556 1012  284  105]\n",
      " [  81   48  153    1]\n",
      " [  40   94  214   73]\n",
      " [  15   39  145   84]]\n",
      "precision and recall on class0 ： 64%   94% \n",
      "\n",
      "precision and recall on class1 ： 16%   4% \n",
      "\n",
      "precision and recall on class2 ： 50%   26% \n",
      "\n",
      "precision and recall on class3 ： 29%   31% \n",
      "\n",
      "total validation samples num ：  4944\n",
      "mean presicion ： 40%\n",
      "mean recall ： 39%\n",
      "mean ap ：  5% \n"
     ]
    }
   ],
   "source": [
    "error_display(test_6,6)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
